#include "./../../../include/online/WebPageQuery/WebQuery.h"
#include "./../../../include/online/WebPageQuery/RedisClient.h"
#include "./../../../include/online/Tools/MyLogger.h"
#include "./../../../include/online/Tools/CppJieba.h"
#include "./../../../include/online/Tools/Configuration.h"

#include <nlohmann/json.hpp>
#include <iostream>
#include <fstream>
#include <algorithm>
#include <iterator>
#include <sstream>

using nlohmann::json;

// 检查是否是UTF-8字符串并修复UTF-8字符串中的无效字符
static int isValidUTF8Character(string *str)
{
    int i, f_size = str->size();
    unsigned char c, c2, c3, c4;
    string to;
    to.reserve(f_size);
    for (i = 0; i < f_size; i++)
    {
        c = (unsigned char)(*str)[i];
        if (c < 32)
        {
            if (c == 9 || c == 10 || c == 13)
            {
                to.append(1, c);
            }
        }
        else if (c < 127)
        {
            to.append(1, c);
        }
        else if (c < 160)
        {
            if (c2 == 128)
            {
                to.append(1, 226);
                to.append(1, 130);
                to.append(1, 172);
            }
            if (c2 == 133)
            {
                to.append(1, 10);
                to.append(1, 13);
            }
        }
        else if (c < 192)
        {
            to.append(1, (unsigned char)194);
            to.append(1, c);
        }
        else if (c < 194)
        {
            to.append(1, (unsigned char)195);
            to.append(1, c - 64);
        }
        else if (c < 224)
        {
            c2 = (unsigned char)(*str)[i + 1];
            if (c2 > 127 && c2 < 192)
            {
                if (c == 194 && c2 < 160)
                {
                    ;
                }
                else
                {
                    to.append(1, c);
                    to.append(1, c2);
                }
                i++;
            }
            else
            {
                to.append(1, (unsigned char)195);
                to.append(1, c - 64);
            }
        }
        else if (c < 240)
        {
            c2 = (unsigned char)(*str)[i + 1];
            c3 = (unsigned char)(*str)[i + 2];
            if (c2 > 127 && c2 < 192 && c3 > 127 && c3 < 192)
            {
                to.append(1, c);
                to.append(1, c2);
                to.append(1, c3);
                i += 2;
            }
            else
            {
                to.append(1, (unsigned char)195);
                to.append(1, c - 64);
            }
        }
        else if (c < 245)
        {
            c2 = (unsigned char)(*str)[i + 1];
            c3 = (unsigned char)(*str)[i + 2];
            c4 = (unsigned char)(*str)[i + 3];
            if (c2 > 127 && c2 < 192 && c3 > 127 && c3 < 192 && c4 > 127 && c4 < 192)
            {
                to.append(1, c);
                to.append(1, c2);
                to.append(1, c3);
                to.append(1, c4);
                i += 3;
            }
            else
            {
                to.append(1, (unsigned char)195);
                to.append(1, c - 64);
            }
        }
        else if (c < 256)
        {
            to.append(1, (unsigned char)195);
            to.append(1, c - 64);
        }
        else
        {
            cout << "WTF? more than 256 values per Byte ? (" << (unsigned int)c << ")" << endl;
        }
    }
    *str = to;
    return 1;
}

// 初始化WebPageQuery对象，加载倒排索引库
WebQuery::WebQuery()
: _dataSql()
{
    setInvertIndexLib();
}

// 释放WebPageQuery对象的资源
WebQuery::~WebQuery()
{

}

// 加载倒排索引库
void WebQuery::setInvertIndexLib()
{
    string filename = Configuration::getInstance()->getMapValue("invertIndex.dat");
    ifstream ifs(filename);
    if (!ifs)
    {
        LogError("unable to open file " + filename);
        return;
    }

    string line;
    while (getline(ifs, line))
    {
        istringstream iss(line);

        string word;
        iss >> word;

        int docid;
        double weight;

        while (iss >> docid)
        {
            iss >> weight;
            _invertIndexLib[word][docid] = weight;
        }
    }
    ifs.close();
    return;
}

// 网页查询
string WebQuery::doQuery(string &queryWord)
{
    LogInfo("开始网页查询");

    // 先查询redis缓存
    json j;
    string result = RedisClient::getInstance()->get(queryWord);
    if (result != "NotFound")
    {
        cout << "在redis缓存中查询到该内容" << endl;
        LogInfo("在redis中查询到网页结果");
        LogInfo("网页查询结束");
        return result;
    }

    // redis缓存中没有查找到，去mysql数据库中查找
    // 将该词组进行分词，过滤掉停用词
    vector<string> words = CppJieba::getInstance()->cut(queryWord);
    Configuration *conf = Configuration::getInstance();

    // 保存过滤掉停用词之后的查询关键词
    vector<string> queryKeyWords;
    queryKeyWords.reserve(words.size());

    for (auto &word : words)
    {
        if (conf->isStopWord(word))
        {
            continue;
        }
        queryKeyWords.push_back(word);
    }

    // 对每个词在哪篇文章出现过，将此篇文章docid进行存储，并求取交集
    // 进行获取到待查询的所有词某一些文章全部出现过，将此篇文章docid进行存储
    getCommonId(queryKeyWords);

    // 获取到每篇文章中的查询词的权重值
    map<int, vector<double>> texts = getTextWeights(queryKeyWords);

    // 计算查询词的权重值，即把查询词当做一篇文章
    vector<double> queryKeyWordsWeight = getQueryWordsWeight(queryKeyWords);

    // 计算每篇文章的余弦相似度,余弦相似度越接近1，说明就是该篇文章
    multimap<double, int> similarity = computeSimlarity(queryKeyWordsWeight, texts);

    // 返回十篇文章，若不足十篇则全部返回；
    vector<string> responseResult;
    if (_commonId.empty() > 10)
    {
        // 编辑计算过后的余弦相似度返回的结果，从后往前，里面有docid
        auto it = similarity.end();
        --it;
        for (int i = 0; i < 10;)
        {
            // 根据docid去mysql数据库中查询
            int docid = it->second;
            WebPage page(queryWebPageByDocId(docid));

            // 对格式化文章进行处理
            page.processDoc();
            if (page.getTitle().size() != 0)
            {
                ++i;
            }
            string summary = page.getSummary(queryKeyWords);
            if (summary.empty())
            {
                it--;
                continue;
            }
            string subResult;
            subResult += "标题：" + page.getTitle() + '\n';
            subResult += "链接：" + page.getDocUrl() + '\n';
            subResult += "内容：" + summary + '\n' + '\n';
            cout << "前str为" << subResult.size();
            isValidUTF8Character(&subResult);
            cout << "后str为" << subResult.size();
            responseResult.push_back(subResult);
            it--;
        }
    }
    else if (_commonId.size() == 0)
    {
        responseResult.push_back("没有匹配的文章");
    }
    else
    {
        for (auto &ele : similarity)
        {
            int docid = ele.second;
            WebPage page(queryWebPageByDocId(docid));

            // 对格式化文章进行处理
            page.processDoc();

            string summary = page.getSummary(queryKeyWords);
            if (summary.empty())
            {
                continue;
            }

            string subResult;
            subResult += "标题：" + page.getTitle() + '\n';
            subResult += "链接：" + page.getDocUrl() + '\n';
            subResult += "内容：" + summary + '\n' + '\n';
            cout << "前str为" << subResult.size();
            isValidUTF8Character(&subResult);
            cout << "后str为" << subResult.size();
            responseResult.push_back(subResult);
        }
    }

    // 将搜索出来的结果插入到redis中
    json j1(responseResult);
    RedisClient::getInstance()->set(queryWord, j1.dump());

    LogInfo("将查询的网页结果放入redis");
    LogInfo("网页查询结束");
    
    return j1.dump();
}

// 获取到去掉停用词后的待查询词在哪些文章中出现过的文章docid
void WebQuery::getCommonId(vector<string> &words)
{
    // 将待查询词在倒排索引库中进行查找
    // 进而找到文章的docid
    for (auto &word : words)
    {
        // 去倒排索引库中进行查找
        auto it = _invertIndexLib.find(word);
        if (it != _invertIndexLib.end()) // 在倒排索引库中找到了
        {
            // 如果交集为空
            if (_commonId.empty())
            {
                // 将该词出现过的所有文章docid，进行插入
                auto it2 = it->second.begin();
                while (it2 != it->second.end())
                {
                    _commonId.insert(it2->first);
                    ++it2;
                }
            }
            else
            {
                // 新建一个存放出现过待查询词的文章docid
                set<int> commonId2;
                auto it2 = it->second.begin();
                while (it2 != it->second.end())
                {
                    commonId2.insert(it2->first);
                    ++it2;
                }

                // 求取交集
                set<int> intersection;
                set_intersection(_commonId.begin(), _commonId.end(),
                                 commonId2.begin(), commonId2.end(),
                                 inserter(intersection, intersection.begin()));

                _commonId = intersection;
            }
        }
    }
    return;
}

// 计算查询词的权重值
// 把查询词当做一篇文章
vector<double> WebQuery::getQueryWordsWeight(vector<string> &words)
{
    vector<double> wordWeights;
    double wordsNum = words.size();
    for (auto &word : words)
    {
        // 统计该词出现过几次
        double freq = count(words.begin(), words.end(), word);
        // 计算该词的权重
        double weight = freq / wordsNum;

        wordWeights.push_back(weight);
        
        // cout << "weight" << weight << endl;
        // cout << "wordsNum" << wordsNum << endl;
        // cout << "wordWeights:" << wordWeights.back() << endl; // 测试
    }
    return wordWeights;
}

// 获取到每篇文章的docid和待查询词的权重,其中所要求的文章张要包含所有待查询关键词
map<int, vector<double>> WebQuery::getTextWeights(vector<string> &words)
{
    map<int, vector<double>> text;
    for (auto &docid : _commonId)
    {
        for (auto &word : words)
        {
            text[docid].push_back(_invertIndexLib[word][docid]);
        }
    }
    return text;
}

// 计算余弦相似度
multimap<double, int> WebQuery::computeSimlarity(vector<double> &queryKeyWordsWeight, map<int, vector<double>> &texts)
{
    // cout << "关键词的权重" << endl;
    // for (auto &ele : queryKeyWordsWeight)
    // {
    //     cout << ele << endl;
    // }

    multimap<double, int> result;
    for (auto &text : texts)
    {
        double sumProducts = 0;
        for (size_t i = 0; i < queryKeyWordsWeight.size(); i++)
        {
            sumProducts += queryKeyWordsWeight[i] * text.second[i];
        }

        long double modelx = 0, modely = 0;
        for (size_t i = 0; i < queryKeyWordsWeight.size(); i++)
        {
            modelx += queryKeyWordsWeight[i] * queryKeyWordsWeight[i];
        }
        modelx = sqrt(modelx);

        for (size_t i = 0; i < text.second.size(); i++)
        {
            modely += text.second[i] * text.second[i];
        }
        modely = sqrt(modely);

        double countResult = sumProducts / modelx / modely;
        result.insert({countResult, text.first});

        // 测试
        // cout << "sumProducts:" << sumProducts << "modelx:" << modelx << "modely:" << modely << endl;
        // cout << "conutResult:" << countResult << endl;
        // cout << "文章号：" << text.first << endl;
    }

    return result;
}

// 根据 docid 从 MySQL 数据库中查询到网页
string WebQuery::queryWebPageByDocId(int docid)
{
    // 校验 docid 是否合法（确保是正整数）
    if (docid <= 0)
    {
        throw std::invalid_argument("Invalid docid: docid must be a positive integer");
    }

    // 构造 SQL 查询语句
    string sql = "SELECT * FROM formatWebpage WHERE docid = " + to_string(docid) + ";";

    // 执行查询
    _dataSql.query(sql);

    // 获取查询结果
    MYSQL_RES *res = _dataSql.store_result();
    if (!res)
    {
        throw std::runtime_error("Failed to store query result.");
    }

    // 获取一行数据
    MYSQL_ROW row = _dataSql.fetch_row(res);
    if (!row)
    {
        mysql_free_result(res);
        throw std::runtime_error("No data found for docid: " + to_string(docid));
    }

    // 获取到网页
    string page = row[1];

    // 释放结果集
    mysql_free_result(res);

    return page;
}